import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.Vector;

public class YuQueCrawler {

    static Logger logger = LoggerFactory.getLogger(YuQueCrawler.class);
    private static String BASE_URL = "https://www.yuque.com/api/catalog_nodes?book_id=44288038";
    private static String replace1 = "https://oss.javaguide.cn";
    private static String replace2 = "https://my-blog-to-use.oss-cn-beijing.aliyuncs.com";
    private static String STATIC_RESOURCE_DIRECTORY = Path.of(System.getProperty("user.dir"), "src/main/resources/yuque").toString();
    private static Vector visited = new Vector<>();
    // 是否覆写文件开关
    private static boolean override = false;

    private static Map map = new HashMap<>() {
        {
            put("yuque_ctoken", "GwjuZmsxPgENQAHJyR5SHo-m");
            put("_yuque_session", "jK7lMKFZjjeZOJbSP_HaXz_q3LhtfvUKwNsK48GE5nWRI7UtcmgA20jv5MQbfQGWvSb40J2CW3RpBkkeaSjhFQ==");
            put("acw_tc", "0bca31c617058328148787518e3126647942b6d4d17c7d59367c9de7feffb6");
            put("_uab_collina", "170225981714739342849013");
            put("lang", "zh-cn");
        }
    };

    public static void main(String[] args) throws IOException, ParseException {
        Connection.Response resp = Jsoup.connect(BASE_URL)
                .timeout(6000)
                .maxBodySize(0)
                .ignoreContentType(true)
                .ignoreHttpErrors(true)
                .cookies(map).execute();
        JSONObject json = (JSONObject) new JSONParser().parse(resp.body());
        JSONArray dataArray = (JSONArray) json.get("data");
        for (Object object : dataArray) {
            JSONObject obj = (JSONObject)object;
            String type = (String) obj.get("type");
            if (type.equals("DOC")) {
                crawPage("https://www.yuque.com/api/docs/" + obj.get("url") + "?book_id=44288038");
            }
        }
        logger.info("Finished.");
    }

    private static void crawPage(String url) {
        try {

            Connection.Response resp = Jsoup.connect(url)
                    .timeout(6000)
                    .maxBodySize(0)
                    .ignoreContentType(true)
                    .ignoreHttpErrors(true)
                    .cookies(map).execute();
            String filePath = getFilePath(url);
            if (resp.contentType().contains("image/")) {
                saveFile(resp.bodyAsBytes(), filePath);
            } else {
                // 保留原字符
                String body = resp.body();
                saveFile(body, filePath);
                // 继续提取URL
                Document doc = Jsoup.parse(body);

                extractURLs(doc.select("link[rel]"));
                extractURLs(doc.select("script[src]"));
                extractURLs(doc.select("img[src]"));
                extractURLs(doc.select("a[href]"));

            }
        } catch (Exception e) {
            logger.error(e.getMessage());
        }
    }

    private static void extractURLs(Elements links) {
        for (Element link : links) {
            // 获取链接地址
            String page = link.attr("src");
            page = page.isEmpty() ? link.attr("href") : page;
            // 黑名单
            if (visited.contains(page) || page.contains("mailto:") || page.contains("#")) {
                continue;
            }
            String url = BASE_URL + page;
            if (page.contains("https")) {
                url = page;
            }
            // 白名单
            if (page.endsWith(".html")
                    || page.endsWith(".png")
                    || page.endsWith(".jpg")
                    || page.endsWith(".svg")
                    || page.endsWith(".js")
                    || page.endsWith(".css")
                    || page.contains("hollis666/vhr2ge")) {
                visited.add(page);
                crawPage(url);
            }
        }
    }

    private static String getFilePath(String url) throws IOException {
        url = url.endsWith("/") ? url + "index.html" : url;
        String fileName = new URL(url).getPath().replace("/", "\\");
        String filePath = STATIC_RESOURCE_DIRECTORY + (fileName.isEmpty() ? "\\index.html" : fileName);
        Path directory = Paths.get(filePath.substring(0, filePath.lastIndexOf('\\')));
        Files.createDirectories(directory);
        return filePath;
    }

    private static void saveFile(String content, String filePath) throws IOException {
        if (override || Files.notExists(Path.of(filePath))) {
            logger.info("Creating File: {}", filePath);
            try (FileOutputStream output = new FileOutputStream(filePath)) {
                output.write(content.replaceAll(replace1, "").replaceAll(replace2, "").getBytes());
            }
        }
    }

    // 图片处理用字节流，防止格式错误不能打开
    private static void saveFile(byte[] content, String filePath) throws IOException {
        if (Files.notExists(Path.of(filePath))) {
            logger.info("Creating Image: {}", filePath);
            try (FileOutputStream output = new FileOutputStream(filePath)) {
                output.write(content);
            }
        }
    }

}